import re

from scrapy import Spider, Selector
from scrapy_redis.spiders import RedisSpider

from slave_requests.items import BossPositionItem, BossCompanyItem


class PositionDetailSpider(RedisSpider):
    name = 'position_detail_spider'
    redis_key = 'dealt_java_position_url'
    select_item = None

    # 工作经验列表
    list_experiences = ['在校生', '应届生', '1年以内', '1-3年', '3-5年', '5-10年', '10年以上']
    # 学历要求列表
    list_degrees = ['大专', '本科', '硕士', '博士']
    # 融资情况列表
    list_financing_situation = ['未融资', '天使轮', 'A轮', 'B轮', 'C轮', 'D轮及以上', '已上市', '不需要融资']
    # 城市拼音映射
    dict_city = {
        '北京': 'beijing', '上海': 'shanghai', '广州': 'guangzhou', '深圳': 'shenzhen', '杭州': 'hangzhou',
        '天津': 'tianjin', '西安': 'xian', '苏州': 'suzhou', '武汉': 'wuhan', '厦门': 'xiamen', '长沙': 'changsha',
        '成都': 'chengdu', '重庆': 'chongqing',
    }
    
    def parse(self, response):
        # tree = etree.HTML(response.text)
        sel = Selector(response)

        # 岗位储存表
        # 公司网址
        company_url = response.url
        # 职位名称
        position_name = sel.xpath('//*[@id="main"]/div[2]/div/div[2]/div[1]/h1/text()').extract_first()
        # 职位所在城市
        messages = sel.xpath('//*[@id="main"]/div[1]/div/div/div[2]/p/text()').extract()
        position_city = None
        # 工作经验
        experience = None
        # 学历要求
        degree = None
        if messages:
            position_city = messages[0]
            for item in messages:
                if item in self.list_experiences:
                    experience = item
                if item in self.list_degrees:
                    degree = item
        # 职位所在城市区
        position_region = ''
        # 职位薪资
        salary = sel.xpath('//*[@id="main"]/div[1]/div/div/div[2]/div[2]/span/text()').extract_first()
        # 岗位信息(描述和要求)
        position_information = ''
        list_information = sel.xpath('//*[@id="main"]/div[3]/div/div[2]/div[2]/div[1]/div/text()').extract()
        pattern1 = re.compile(r'[\r\n]+\s*')
        if list_information:
            for item in list_information:
                item = pattern1.sub(':!:', item)
                position_information += item + ':!:'
        # 职位发布时间
        pattern2 = re.compile(r'"upDate":  "(.*?)T')
        position_putime = pattern2.findall(response.text)[0] if pattern2.findall(response.text) else None
        # 职位标签
        position_labels_01 = sel.xpath('//*[@id="main"]/div[3]/div/div[2]/div[2]/div[2]/div[2]/span/text()').extract()
        position_labels_02 = sel.xpath('//*[@id="main"]/div[1]/div/div/div[2]/div[3]/div[2]/span/text()').extract()
        position_labels = list(set(position_labels_01 + position_labels_02))

        # 公司表存储字段
        # 公司网址
        # 处理信息不全的情况-主要信息
        temp_index1 = 5
        temp_index2 = 6
        if not sel.xpath('//*[@id="main"]/div[3]/div/div[2]/div[2]/div[6]/div/div[1]/text()').extract():
            temp_index1 -= 1
            temp_index2 -= 1
            if not sel.xpath('//*[@id="main"]/div[3]/div/div[2]/div[2]/div[5]/div[1]/text()').extract():
                temp_index1 -= 1
                temp_index2 -= 1
        # 公司名
        company_name = sel.xpath(
            f'//*[@id="main"]/div[3]/div/div[2]/div[2]/div[{temp_index1}]/div[1]/text()').extract_first()
        # 公司地址
        company_address = sel.xpath(
            f'//*[@id="main"]/div[3]/div/div[2]/div[2]/div[{temp_index2}]/div/div[1]/text()').extract_first()
        # 公司性质
        company_type = sel.xpath(
            f'//*[@id="main"]/div[3]/div/div[2]/div[2]/div[{temp_index1}]/div[2]/li[4]/text()').extract_first()
        # 融资情况
        financing_situation = sel.xpath('//*[@id="main"]/div[3]/div/div[1]/div[2]/p[2]/text()').extract_first()
        # 处理信息不全的情况-次
        if financing_situation not in self.list_financing_situation:
            company_number = financing_situation
            financing_situation = None
        else:
            # 规模人数
            company_number = sel.xpath('//*[@id="main"]/div[3]/div/div[1]/div[2]/p[3]/text()').extract_first()

        print('*' * 50)
        print('公司网址:', company_url)
        print('岗位名称:', position_name)
        print('岗位所在城市:', position_city)
        print('工作经验:', experience)
        print('学历要求:', degree)
        print('岗位所在城市所属区:', position_region)
        print('薪资:', salary)
        print('岗位信息:', position_information)
        print('岗位发布时间:', position_putime)
        print('岗位标签:', position_labels)

        print('=' * 50)
        print('公司名称:', company_name)
        print('公司网址:', company_url)
        print('公司地址:', company_address)
        print('公司人数:', company_number)
        print('公司性质:', company_type)
        print('融资情况:', financing_situation)
        print('信息打印完成')

        item = BossPositionItem()
        # item['type'] = 'position'
        item['subject'] = 'java'
        item['city'] = self.dict_city[position_city]
        item['company_url'] = company_url
        item['position_name'] = position_name
        item['position_city'] = position_city
        item['experience'] = experience
        item['degree'] = degree
        item['position_region'] = position_region
        item['salary'] = salary
        item['position_information'] = position_information
        item['position_putime'] = position_putime
        item['position_labels'] = position_labels
        item['company_name'] = company_name
        item['company_address'] = company_address
        item['company_number'] = company_number
        item['company_type'] = company_type
        item['financing_situation'] = financing_situation
        self.select_item = 'position'
        
        yield item
        
        # item2 = BossCompanyItem()
        # item['type'] = 'company'
        # item['subject'] = 'java'
        # item['city'] = 'beijing'
        # item2['company_name'] = company_name
        # item2['company_url'] = company_url
        # item2['company_address'] = company_address
        # item2['company_number'] = company_number
        # item2['company_type'] = company_type
        # item2['financing_situation'] = financing_situation
        # self.select_item = 'company'
        #
        # yield item2
